# -*- coding:utf-8 -*-
"""
@Project :   PyCharm
@File    :   MCD12Q1.py
@Time    :   2021/10/20 09:16 下午
@Author  :   lsw
@Version :   1.0
@Contact :   shi_weihappy@126.com
@License :   (C)Copyright 2020-2021, lsw
@Desc    :   None
1、转投影 EPSG:3857
2、切瓦片，上传数据到S3
3、生成缩略图
4、写入ES数据
"""
import shutil
import os
import json
import datetime
import requests
from osgeo import gdal, osr, gdalconst
import lz4.frame
import numpy as np
import logging
import math
import time
from boto3.session import Session
from botocore.client import Config

logging.basicConfig(level = logging.INFO,format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)

logging.getLogger('boto3').setLevel(logging.ERROR)
logging.getLogger('botocore').setLevel(logging.ERROR)
logging.getLogger('nose').setLevel(logging.ERROR)
logging.getLogger('s3transfer').setLevel(logging.ERROR)
logging.getLogger('urllib3').setLevel(logging.ERROR)

gdal.UseExceptions()

# endPoint: minio节点
# access_key_id: minio的key_id
# secret_access_key: minio的access_key
# tile_bucket: 瓦片的bucket
# preview_bucket: 缩略图的buket
# es_server_url: es的访问路径
# tif_path: 原始文件的路径
# s3_key: 数据的索引ID
# engine.mcd12q1_v006.image 是需要建立的ES表名称，也就是建立json文件的名字
PIEConfig = {
    "endPoint": "http://10.25.21.3:32309/",
    "access_key_id": "minio",
    "secret_access_key": "minio123",
    "tile_bucket": "pie-engine-default-tiles",
    "preview_bucket": "pie-engine-preview-images",
    "es_server_url": "http://10.25.21.4:30013/engine.fy3d_mersi_1000m_d.image/_doc/",
     "es_auth": "ZWxhc3RpYzpodGh0QDIwMjI=",
    "tif_path":"/Batchprocess/fy3d_l1_1000m_d/result/",
    "s3_key":"NSMC/FY3D/MERSI/L1/1000M/D"
}


class AWSTool(object):
    def __init__(self, bucket):
        super(AWSTool, self).__init__()
        self._bucket = bucket
        self._access_key_id = PIEConfig["access_key_id"]
        self._secret_access_key = PIEConfig["secret_access_key"]
        self._endpoint = PIEConfig["endPoint"]

    def getBucket(self):
        return self._bucket

    def initS3Client(self):
        session = Session(aws_access_key_id=self._access_key_id,
                          aws_secret_access_key=self._secret_access_key)
        s3_config = Config(max_pool_connections=20, retries={'max_attempts': 20})
        s3 = session.client('s3', config=s3_config, endpoint_url=self._endpoint)
        return s3

    @classmethod
    def deleteFileByKey(cls, s3Bucket, keys):
        """
        :param cls:
        :param s3Bucket:
        :param keys:
        :return:
        """
        _objs = []
        for key in keys:
            _objs.append({
                "Key": key
            })
        s3Bucket.delete_objects(
            Delete={
                "Objects": _objs
            }
        )

    @classmethod
    def listS3BucketPrefix(cls, s3, bucket, prefix):
        """
        返回AWS S3 bucket的列表
        :param s3:
        :param bucket:
        :param prefix:
        :return:
        """
        try:
            pag = s3.get_paginator('list_objects_v2')
            params = {
                'Bucket': bucket,
                'Prefix': prefix,
                'Delimiter': '/'
            }
            directories = []
            response_iterator = pag.paginate(**params)
            for subset in response_iterator:
                if 'CommonPrefixes' in subset.keys():
                    directories.extend(subset.get('CommonPrefixes'))
            return [r['Prefix'] for r in directories]
        except Exception as e:
            logger.info("listS3Bucket error is: {}".format(e))
            return None

    @classmethod
    def listS3BucketFile(cls, s3, bucket, prefix):
        """
        返回AWS S3 bucket 文件列表
        :param s3:
        :param bucket:
        :param prefix:
        :return:
        """
        try:
            paginator = s3.get_paginator('list_objects_v2')
            _des_file_list = []
            response_iterator = paginator.paginate(
                Bucket=bucket,
                Prefix=prefix
            )
            for page in response_iterator:
                if "Contents" in page:
                    for n in page["Contents"]:
                        key = n["Key"]
                        _des_file_list.append({
                            "key": key,
                            "size": n["Size"]
                        })
            return _des_file_list
        except Exception as e:
            logger.info("get s3 file list fail, error is: {}".format(e))
            return None

    @classmethod
    def delS3BucketFile(cls, s3, bucket, prefix, s3Bucket):
        """
        删除指定的AWS S3 bucket 文件列表
        :param s3:
        :param bucket:
        :param prefix:
        :param s3Bucket:
        :return:
        """
        try:
            paginator = s3.get_paginator('list_objects_v2')
            response_iterator = paginator.paginate(
                Bucket=bucket,
                Prefix=prefix
            )
            index = 0
            for page in response_iterator:
                if "Contents" in page:
                    delete_keys = []
                    index += 1
                    for n in page["Contents"]:
                        delete_keys.append(n["Key"])
                    cls.deleteFileByKey(s3Bucket, delete_keys)
        except Exception as e:
            logger.info("get s3 file list fail, error is: {}".format(e))

    @classmethod
    def getS3FileContent(cls, s3, bucket, key):
        """
        获取指定文件内容
        :param s3:
        :param bucket:
        :param key:
        :return:
        """
        try:
            params = {
                'Bucket': bucket,
                'Key': key
            }
            response = s3.get_object(**params)
            return response['Body'].read()
        except Exception as e:
            logger.info("getS3FileContent, error is: {}".format(e))
            return None

class MercatorTools(object):
    def __init__(self, tileSize=256):
        """
        Initialize the TMS Global Mercator pyramid
        :param tileSize:
        """
        self.tileSize = tileSize
        self.initialResolution = 2 * math.pi * 6378137 / self.tileSize
        # 156543.03392804062 for tileSize 256 pixels
        self.originShift = 2 * math.pi * 6378137 / 2.0
        self.MAXZOOMLEVEL = 32

    def LatLonToMeters(self, lat, lon):
        """
        Converts given lat/lon in WGS84 Datum to XY in Spherical Mercator EPSG:3857
        :param lat:
        :param lon:
        :return:
        """
        mx = lon * self.originShift / 180.0
        my = math.log(math.tan((90 + lat) * math.pi / 360.0)) / (math.pi / 180.0)
        my = my * self.originShift / 180.0
        return mx, my

    def MetersToLatLon(self, mx, my):
        """
        Converts XY point from Spherical Mercator EPSG:3857 to lat/lon in WGS84 Datum
        :param mx:
        :param my:
        :return:
        """
        lon = (mx / self.originShift) * 180.0
        lat = (my / self.originShift) * 180.0

        lat = 180 / math.pi * \
              (2 * math.atan(math.exp(lat * math.pi / 180.0)) - math.pi / 2.0)
        return lat, lon

    def PixelsToMeters(self, px, py, zoom):
        """
        Converts pixel coordinates in given zoom level of pyramid to EPSG:3857
        :param px:
        :param py:
        :param zoom:
        :return:
        """
        res = self.Resolution(zoom)
        mx = px * res - self.originShift
        my = py * res - self.originShift
        return mx, my

    def MetersToPixels(self, mx, my, zoom):
        """
        Converts EPSG:3857 to pyramid pixel coordinates in given zoom level
        :param mx:
        :param my:
        :param zoom:
        :return:
        """
        res = self.Resolution(zoom)
        px = (mx + self.originShift) / res
        py = (my + self.originShift) / res
        return px, py

    def PixelsToTile(self, px, py):
        """
        Returns a tile covering region in given pixel coordinates
        :param px:
        :param py:
        :return:
        """
        tx = int(math.ceil(px / float(self.tileSize)) - 1)
        ty = int(math.ceil(py / float(self.tileSize)) - 1)
        return tx, ty

    def PixelsToRaster(self, px, py, zoom):
        """
        Move the origin of pixel coordinates to top-left corner
        :param px:
        :param py:
        :param zoom:
        :return:
        """
        mapSize = self.tileSize << zoom
        return px, mapSize - py

    def MetersToTile(self, mx, my, zoom):
        """
        Returns tile for given mercator coordinates
        :param mx:
        :param my:
        :param zoom:
        :return:
        """
        px, py = self.MetersToPixels(mx, my, zoom)
        return self.PixelsToTile(px, py)

    def TileBounds(self, tx, ty, zoom):
        """
        Returns bounds of the given tile in EPSG:3857 coordinates
        :param tx:
        :param ty:
        :param zoom:
        :return:
        """
        minx, miny = self.PixelsToMeters(
            tx * self.tileSize, ty * self.tileSize, zoom)
        maxx, maxy = self.PixelsToMeters(
            (tx + 1) * self.tileSize, (ty + 1) * self.tileSize, zoom)
        return minx, miny, maxx, maxy

    def TileLatLonBounds(self, tx, ty, zoom):
        """
        Returns bounds of the given tile in latitude/longitude using WGS84 datum
        :param tx:
        :param ty:
        :param zoom:
        :return:
        """
        bounds = self.TileBounds(tx, ty, zoom)
        minLat, minLon = self.MetersToLatLon(bounds[0], bounds[1])
        maxLat, maxLon = self.MetersToLatLon(bounds[2], bounds[3])
        return minLat, minLon, maxLat, maxLon

    def Resolution(self, zoom):
        """
        Resolution (meters/pixel) for given zoom level (measured at Equator)
        :param zoom:
        :return:
        """
        return self.initialResolution / (2 ** zoom)

    def ZoomForPixelSize(self, pixelSize):
        """
        通过原始像素的分辨率获取最大缩放级别
        :param pixelSize:
        :return:
        """
        _value = 0
        _zoom = 0
        _resolutions = []
        for i in range(self.MAXZOOMLEVEL):
            _resolutions.append(self.Resolution(i))
        _differ = list(map(lambda x: math.fabs(x - pixelSize), _resolutions))
        for i in range(len(_differ)):
            if i == 0:
                _value = _differ[i]
                _zoom = i
            else:
                if _value >= _differ[i]:
                    _value = _differ[i]
                    _zoom = i
        return _zoom

def getDateByJulinDay(year_julianday):
    year_julianday = str(year_julianday)
    year = int(year_julianday[:4])
    day = int(year_julianday[-2:])
    month = int(year_julianday[-4:-2])
    first_day = datetime.datetime(year, month, 1)
    _day = first_day + datetime.timedelta(day - 1)
    _day = datetime.datetime.strftime(_day, '%Y-%m-%d')
    return _day

def walkDirFile(srcPath, ext=".tif"):
    """
    遍历文件夹
    :param srcPath:
    :param ext:
    :return:
    """
    if not os.path.exists(srcPath):
        logger.info("not find path:{0}".format(srcPath))
        return None
    if os.path.isfile(srcPath):
        return None

    if os.path.isdir(srcPath):
        fileList = []
        for root, dirs, files in os.walk(srcPath):
            for name in files:
                filePath = os.path.join(root, name)
                if ext:
                    if ext == os.path.splitext(name)[1]:
                        fileList.append(filePath)
                else:
                    fileList.append(filePath)
        fileList.sort()
        return fileList
    else:
        return None

def getRasterBandInfo(rasterFileName, bandNumber=1):
    """
    获得影像的信息
    :param rasterFileName:
    :param bandNumber:
    :return:
    """
    try:
        raster = gdal.Open(rasterFileName)
        band = raster.GetRasterBand(bandNumber)
        geotransform = raster.GetGeoTransform()
        originX = geotransform[0]
        originY = geotransform[3]
        pixelWidth = geotransform[1]
        pixelHeight = geotransform[5]
        rtnX = geotransform[2]
        rtnY = geotransform[4]
        cols = raster.RasterXSize
        rows = raster.RasterYSize
        ominx = originX
        omaxx = originX + cols * pixelWidth
        omaxy = originY
        ominy = originY + rows * pixelHeight
        logger.info("=============rasterFileName = {0} ================".format(rasterFileName))
        logger.info("rtnX = {0}, rtnY = {1}".format(rtnX, rtnY))
        logger.info("originX = {0}, originY = {1}".format(originX, originY))
        logger.info("pixelWidth = {0}, pixelHeight = {1}".format(pixelWidth, pixelHeight))
        logger.info("cols = {0}, rows = {1}".format(cols, rows))
        logger.info(f"{ominx} {omaxx} {ominy} {omaxy}")
        bandArray = band.ReadAsArray()
        logger.info("=======================end==========================")
        return {
            "array": bandArray,
            "geotransform": geotransform,
            "cols": cols,
            "rows": rows,
            "pixelWidth": pixelWidth,
            "pixelHeight": pixelHeight
        }
    except RuntimeError as e:
        logger.info("can not open rasterFile : {0}, error is {1}".format(rasterFileName, e))
        return None

def generateSingleTifByArray(in_tif, in_array, out_tif, invalid_value=-1, dtype=gdal.GDT_Float32, bandNum=1):
    """
    通过数组生成tif
    :param in_tif:
    :param in_array:
    :param out_tif:
    :param invalid_value:
    :param dtype:
    :param bandNum:
    :return:
    """
    if os.path.exists(out_tif):
        os.remove(out_tif)

    out_tif_path = os.path.dirname(out_tif)
    if not os.path.exists(out_tif_path):
        os.makedirs(out_tif_path)
    try:
        raster = gdal.Open(in_tif)
    except RuntimeError as e:
        logger.info("can not open rasterFile : {0}, error is {1}".format(in_tif, e))
        return False
    geotransform = raster.GetGeoTransform()
    originX = geotransform[0]
    originY = geotransform[3]
    pixelWidth = geotransform[1]
    pixelHeight = geotransform[5]
    cols = raster.RasterXSize
    rows = raster.RasterYSize

    driver = gdal.GetDriverByName('GTiff')
    outRaster = driver.Create(out_tif, cols, rows, 1, dtype, options=["COMPRESS=LZW"])
    outRaster.SetGeoTransform((originX, pixelWidth, 0, originY, 0, pixelHeight))
    outband = outRaster.GetRasterBand(1)
    outRasterSRS = osr.SpatialReference()
    outRasterSRS.ImportFromWkt(raster.GetProjectionRef())
    outRaster.SetProjection(outRasterSRS.ExportToWkt())
    if invalid_value is not None:
        outband.Fill(invalid_value)
        outband.SetNoDataValue(invalid_value)

    band = raster.GetRasterBand(bandNum)
    # blockSize = band.GetBlockSize()
    # xBlockSize = blockSize[0]
    # yBlockSize = blockSize[1]
    xBlockSize = 256
    yBlockSize = 256
    for i in range(0, rows, yBlockSize):
        if i + yBlockSize < rows:
            numRows = yBlockSize
        else:
            numRows = rows - i
        for j in range(0, cols, xBlockSize):
            if j + xBlockSize < cols:
                numCols = xBlockSize
            else:
                numCols = cols - j
            resultArray = in_array[i:i + numRows, j:j + numCols]
            if resultArray is None:
                srcRasterArray = band.ReadAsArray(j, i, numCols, numRows)
                resultArray = np.ones(srcRasterArray.shape) * invalid_value
            outband.WriteArray(resultArray, j, i)
    outband.FlushCache()

def geo_query(ds, ulx, uly, lrx, lry, querysize = 0):
    """
    For given dataset and query in cartographic coordinates
    returns parameters for ReadRaster() in raster coordinates and
    x/y shifts (for border tiles). If the querysize is not given, the
    extent is returned in the native resolution of dataset ds.
    :param ds:
    :param ulx:
    :param uly:
    :param lrx:
    :param lry:
    :param querysize:
    :return:
    """
    geotran = ds.GetGeoTransform()
    rx= int((ulx - geotran[0]) / geotran[1] + 0.001)
    ry= int((uly - geotran[3]) / geotran[5] + 0.001)
    # TODO
    rxsize = max(1, math.ceil((lrx - ulx) / geotran[1]))
    rysize = max(1, math.ceil((lry - uly) / geotran[5]))

    if not querysize:
        wxsize, wysize = rxsize, rysize
    else:
        wxsize, wysize = querysize, querysize

    # Coordinates should not go out of the bounds of the raster
    wx = 0
    if rx < 0:
        rxshift = abs(rx)
        wx = int( wxsize * (float(rxshift) / rxsize) )
        wxsize = wxsize - wx
        rxsize = rxsize - int( rxsize * (float(rxshift) / rxsize) )
        rx = 0
    if rx+rxsize > ds.RasterXSize:
        wxsize = math.ceil( wxsize * (float(ds.RasterXSize - rx) / rxsize) )
        rxsize = ds.RasterXSize - rx
    # TODO: 解决边界为0的情况，这种情况是比较特殊的情况
    if wxsize == 0:
        wxsize = 1
        wx -= 1
        if wx <=0 :
            wx = 0

    wy = 0
    if ry < 0:
        ryshift = abs(ry)
        wy = math.ceil( wysize * (float(ryshift) / rysize) )
        wysize = wysize - wy
        rysize = rysize - int( rysize * (float(ryshift) / rysize) )
        ry = 0
    if ry+rysize > ds.RasterYSize:
        wysize = math.ceil( wysize * float(ds.RasterYSize - ry) / rysize )
        rysize = ds.RasterYSize - ry
    # TODO: 解决边界为0的情况，这种情况是比较特殊的情况
    if wysize == 0:
        wysize = 1
        wy -= 1
        if wy <=0 :
            wy = 0
    return (rx, ry, rxsize, rysize), (wx, wy, wxsize, wysize)

def compressData(data):
    return lz4.frame.compress(data)

def decompressData(data):
    return lz4.frame.decompress(data)

def generateTileToS3(s3, bucket, key, tif, epsg=3857, compress=True, nodata=None):
    mercator = MercatorTools()
    try:
        out_ds = gdal.Open(tif)
    except RuntimeError as e:
        logger.info("can not open rasterFile : {0}, error is {1}".format(tif, e))
        return

    rasterCount = out_ds.RasterCount
    out_gt = out_ds.GetGeoTransform()
    ominx = out_gt[0]
    omaxx = out_gt[0] + out_ds.RasterXSize * out_gt[1]
    omaxy = out_gt[3]
    ominy = out_gt[3] - out_ds.RasterYSize * out_gt[1]
    _pixel_width = out_gt[1]
    _pixel_height = out_gt[5]
    _pixel = max(_pixel_width, _pixel_height)
    _zoom = mercator.ZoomForPixelSize(_pixel)
    logger.info(f"process {tif} , zoom: {_zoom}")
    # 由于处理完成的数据名称是 FY3D_MERSI_WHOLE_GLL_L1_20230514_0850_1000M_B1.tif，这个不需要二次处理。
    _file_band_name = os.path.splitext(os.path.basename(tif))[0]
    # _file_names = _file_name.split("_")
    #_file_band_name = f"{'_'.join(_file_names[:len(_file_names) - 1])}_{_file_names[-1]}"
    # _file_band_name = f"{'_'.join(_file_names[:len(_file_names) - 1])}_{_file_names[-1]}_{_file_names[-1]}"
    for _band in range(1, rasterCount + 1):
        logger.info("process --------- {}".format(_band))
        out_band = out_ds.GetRasterBand(_band)
        noDataValue = out_band.GetNoDataValue()
        dataType = out_band.DataType
        tile_size = 256
        for tz in range(_zoom + 1):
            tminx, tminy = mercator.MetersToTile(ominx, ominy, tz)
            tmaxx, tmaxy = mercator.MetersToTile(omaxx, omaxy, tz)
            # crop tiles extending world limits (+-180,+-90)
            tminx, tminy = max(0, tminx), max(0, tminy)
            tmaxx, tmaxy = min(2 ** tz - 1, tmaxx), min(2 ** tz - 1, tmaxy)
            _count = (tmaxy + 1 - tminy) * (tmaxx + 1 - tminx)
            logger.info(f"zoom: {tz}, tile count: {_count}")
            for ty in range(tmaxy, tminy - 1, -1):
                for tx in range(tminx, tmaxx + 1):
                    # Tile bounds in EPSG:3857
                    b = mercator.TileBounds(tx, ty, tz)
                    # Tile bounds in raster coordinates for ReadRaster query
                    rb, wb = geo_query(out_ds, b[0], b[3], b[2], b[1], querysize=tile_size)
                    rx, ry, rxsize, rysize = rb
                    wx, wy, wxsize, wysize = wb
                    if rxsize == 0 or rysize == 0:
                        continue
                    if tz < 6:
                        resample_alg = gdalconst.GRIORA_NearestNeighbour
                    else:
                        resample_alg = gdalconst.GRIORA_Average
                    data = out_ds.ReadRaster(
                        xoff=rx,
                        yoff=ry,
                        xsize=rxsize,
                        ysize=rysize,
                        buf_xsize=wxsize,
                        buf_ysize=wysize,
                        band_list=[_band],
                        resample_alg=resample_alg,
                    )
                    # TODO：内存生成
                    dataset = gdal.GetDriverByName('MEM').Create('', tile_size, tile_size, 1, dataType)

                    #EPSG:3857
                    minx, miny, maxx, maxy = mercator.TileBounds(tx, ty, tz)
                    geotrans = (minx, (maxy - miny) / tile_size, 0.0, maxy, 0.0, -(maxx - minx) / tile_size)

                    dataset.SetGeoTransform(geotrans)
                    srs = osr.SpatialReference()
                    srs.ImportFromEPSG(epsg)
                    dataset.SetProjection(srs.ExportToWkt())
                    if nodata is not None:
                        dataset.GetRasterBand(1).Fill(nodata)
                        dataset.GetRasterBand(1).SetNoDataValue(nodata)
                    else:
                        if noDataValue is not None:
                            dataset.GetRasterBand(1).Fill(noDataValue)
                            dataset.GetRasterBand(1).SetNoDataValue(noDataValue)

                    dataset.WriteRaster(
                        xoff=wx,
                        yoff=wy,
                        xsize=wxsize,
                        ysize=wysize,
                        buf_string=data,
                        band_list=[1]
                    )
                    # TODO:所有空的瓦片全部删除
                    bandArray = dataset.GetRasterBand(1).ReadAsArray()
                    if np.nansum(bandArray != nodata) <= 0:
                        del dataset
                        continue
                    # 读取
                    data = dataset.ReadRaster()
                    if not compress:
                        _data = b"bitmap\x00\x00" + data
                    else:
                        _data = b"lz4\x00" + compressData(data)
                    _key = f"{key}/{_file_band_name}_{tz}_{(2 ** tz - 1) - ty}_{tx}"
                    s3.put_object(
                        Body=_data,
                        Bucket=bucket,
                        Key=_key,
                        StorageClass="STANDARD"
                    )
                    del dataset

def transformTif(src_tif, dest_tif):
    """
    影像转投影
    :param src_tif:
    :param dest_tif:
    :return:
    """
    _dest_path = os.path.dirname(dest_tif)
    if not os.path.exists(_dest_path):
        os.makedirs(_dest_path)
    _tif_name = os.path.basename(src_tif)
    if os.path.exists(dest_tif):
        os.remove(dest_tif)
    cmd_str = f"gdalwarp -co COMPRESS=LZW -co BIGTIFF=YES -t_srs EPSG:3857 {src_tif} {dest_tif}"
    os.system(cmd_str)

def getMaxZoomByPixel(pixel):
    """
    通过原始像素的分辨率获取最大缩放级别
    :param pixel:
    :return:
    """
    _value = 0
    _zoom = 0
    RESOLUTIONS = [
        156543.03392800014,
        78271.51696399994,
        39135.75848200009,
        19567.87924099992,
        9783.93962049996,
        4891.96981024998,
        2445.98490512499,
        1222.992452562495,
        611.4962262813797,
        305.74811314055756,
        152.87405657041106,
        76.43702828507324,
        38.21851414253662,
        19.10925707126831,
        9.554628535634155,
        4.77731426794937,
        2.388657133974685,
        1.1943285668550503,
        0.5971642835598172,
        0.29858214164761665
    ]
    _differ = list(map(lambda x: math.fabs(x - pixel), RESOLUTIONS))
    for i in range(len(_differ)):
        if i == 0:
            _value = _differ[i]
            _zoom = i
        else:
            if _value >= _differ[i]:
                _value = _differ[i]
                _zoom = i
    return _zoom

def writeToES(files, key, bucket):
    metaData = {
        "name": "",
        "time": "",
        "size": "0GB",
        "uri": f"bucket:{bucket}",
        "bands": [],
        "centre": {},
        "geom": "",
        "properties": {}
    }
    meta_size = 0
    for file in files:
        try:
            dataset = gdal.Open(file)
            count = dataset.RasterCount
            nXSize = dataset.RasterXSize
            nYSize = dataset.RasterYSize
            geoTransform = dataset.GetGeoTransform()
            xMin = geoTransform[0]
            yMax = geoTransform[3]
            xCell = geoTransform[1]
            yCell = geoTransform[5]
            xMax = xMin + xCell * nXSize
            yMin = yMax + yCell * nYSize
            maxCell = max(abs(xCell), abs(yCell))

            xMin_3857 = xMin
            xMax_3857 = xMax
            yMin_3857 = yMin
            yMax_3857 = yMax

            source = osr.SpatialReference()
            source.ImportFromEPSG(3857)
            target = osr.SpatialReference()
            target.ImportFromEPSG(4326)

            coordTrans = osr.CoordinateTransformation(source, target)
            transXY = coordTrans.TransformPoint(xMin, yMin)
            #gdal2.x和gdal3.x是反着的
            xMin = transXY[1]
            yMin = transXY[0]
            transXY = coordTrans.TransformPoint(xMax, yMax)
            xMax = transXY[1]
            yMax = transXY[0]
            sourceCS = "EPSG:3857"
            min_level = 0
            max_level = getMaxZoomByPixel(maxCell)
            band_name = os.path.splitext(os.path.basename(file))[0].split("_")[-1]

            img_Band = dataset.GetRasterBand(1)
            _type = gdal.GetDataTypeName(img_Band.DataType)
            nodata = img_Band.GetNoDataValue()
            if nodata is not None:
                nodata = np.float64("{:.19}".format(np.float32(nodata)))
            try:
                min_value, max_value = img_Band.ComputeRasterMinMax()
            except Exception as e:
                min_value = max_value = nodata
                logger.info("get min max value error: {}".format(e))

            band = {
                "name": band_name,
                "type": _type,
                "crs": sourceCS,
                "extent": "{0},{1},{2},{3}".format(xMin_3857, yMax_3857, xMax_3857, yMin_3857),
                "noData": nodata,
                "xsize": nXSize,
                "ysize": nYSize,
                "min": min_value,
                "max": max_value,
                "minLevel": min_level,
                "maxLevel": max_level
            }

            metaData["bands"].append(band)

            if not metaData['centre']:
                center_lon = (xMin + xMax) / 2
                center_lat = (yMin + yMax) / 2
                centroid = f"POINT ({center_lon} {center_lat})"
                metaData['centre'] = centroid

            if not metaData['geom']:
                geom = f"POLYGON (({xMin} {yMin},{xMax} {yMin},{xMax} {yMax},{xMin} {yMax},{xMin} {yMin}))"
                metaData["geom"] = geom
            del dataset
        except RuntimeError as e:
            logger.info(f"error is: {e}")
            return

        fileProperties = os.stat(file)
        meta_size += fileProperties.st_size
    meta_size = meta_size / 1024
    if meta_size < 1024:
        meta_size = "%.3f" % meta_size
        meta_size = "{0}KB".format(meta_size)
    else:
        meta_size = meta_size / 1024
        if meta_size < 1024:
            meta_size = "%.3f" % meta_size
            meta_size = "{0}MB".format(meta_size)
        else:
            meta_size = meta_size / 1024
            meta_size = "%.3f" % meta_size
            meta_size = "{0}GB".format(meta_size)
    metaData["size"] = meta_size
    metaData["name"] = key
    # TODO:时间、属性根据不同数据获取 MCD12Q1.A2020001.h25v05.006.2021360015257.hdf  FY3D_MERSI_WHOLE_GLL_L1_20230203_0015_1000M_C001.tif
    logger.info(key)
    # TODO: 更新影像日期为年月日 时分秒
    _keys = key.split("/")[-1].strip().split("_")
    _date_str = f"{_keys[5]}{_keys[6]}"
    # _time = getDateByJulinDay(julin_date)
    _image_date = datetime.datetime.strptime(_date_str, "%Y%m%d%H%M")
    _time = _image_date.strftime('%Y-%m-%dT%H:%M:%SZ')
    metaData["time"] = _time
    metaData["properties"] = {
        "date": _time,
        "year": int(_date_str[:4])
    }
    logger.info("meta :{}".format(metaData))

    #TODO：替换为指定的ES路径
    _name = metaData["name"].replace("/", "%2F")
    es_server_url = f"{PIEConfig['es_server_url']}{_name}"
    payload = json.dumps(metaData)
    headers = {'Content-Type': 'application/json',
               'Authorization': 'Basic {0}'.format(PIEConfig["es_auth"])}
    response = requests.request("POST", es_server_url, headers=headers, data=payload)
    result = json.loads(response.text)
    logger.info(result)

def generateRGBFile(srcFile, jpgFileName):
    """
    生产 缩略图 jpg 文件
    :param jpgFileName:
    :param srcFile
    :return:
    """
    if not os.path.exists(srcFile):
        logger.info("generateJPGFile: srcFile {} file is not exist!".format(srcFile))
        return

    _src_path = os.path.dirname(srcFile)
    _vrt_file = os.path.join(_src_path, "{}.vrt".format(jpgFileName))
    _jpg_file = os.path.join(_src_path, jpgFileName)

    # 生成虚拟文件vrt
    if os.path.exists(_vrt_file):
        os.remove(_vrt_file)
    cmd_str = "gdalbuildvrt -b 1 -b 2 -b 3 -r bilinear {vrt} {src}".format(
        vrt=_vrt_file,
        src=srcFile
    )
    os.system(cmd_str)

    #生成缩略图
    if os.path.exists(_jpg_file):
        os.remove(_jpg_file)
    cmd_str = "gdal_translate -of JPEG -ot Byte -scale -outsize 200 200 {vrt} {jpg}".format(
        vrt=_vrt_file,
        jpg=_jpg_file
    )
    os.system(cmd_str)

    #删除虚拟文件
    if os.path.exists(_vrt_file):
        os.remove(_vrt_file)
    return _jpg_file

def generateGrayFile(srcFile, jpgFileName):
    """
    生产 缩略图 jpg 文件
    :param jpgFileName:
    :param srcFile:
    :return:
    """
    if not os.path.exists(srcFile):
        logger.info("generateJPGFile: srcFile {} file is not exist!".format(srcFile))
        return

    _src_path = os.path.dirname(srcFile)
    _vrt_file = os.path.join(_src_path, "{}.vrt".format(jpgFileName))
    _jpg_file = os.path.join(_src_path, jpgFileName)

    # 生成虚拟文件vrt
    if os.path.exists(_vrt_file):
        os.remove(_vrt_file)
    cmd_str = "gdalbuildvrt -b 1 -r bilinear {vrt} {src}".format(
        vrt=_vrt_file,
        src=srcFile
    )
    os.system(cmd_str)

    #生成缩略图
    if os.path.exists(_jpg_file):
        os.remove(_jpg_file)
    cmd_str = "gdal_translate -of JPEG -ot Byte -scale -outsize 200 200 {vrt} {jpg}".format(
        vrt=_vrt_file,
        jpg=_jpg_file
    )
    os.system(cmd_str)

    #删除虚拟文件
    if os.path.exists(_vrt_file):
        os.remove(_vrt_file)
    return _jpg_file

def generatePreviewAndUploadToS3(src, s3, bucket, key):
    """
    生成缩略图并且上传文件到S3中
    :param s3:
    :param bucket:
    :param src:
    :param key:
    :return:
    """
    if not os.path.exists(src):
        logger.info("error: {} is not exits".format(src))
        return
    t1 = time.time()
    if not s3 \
        or not bucket \
        or not key:
        logger.info("generatePreviewAndUploadToS3: param is error, key is {}".format(key))
        return
    _jpgName = os.path.splitext(os.path.basename(src))[0] + ".jpg"
    _key = os.path.join(key, _jpgName)
    raster = gdal.Open(src)
    bandCount = raster.RasterCount
    if bandCount >= 3:
        _jpg_file = generateRGBFile(srcFile=src, jpgFileName=_jpgName)
    else:
        _jpg_file = generateGrayFile(srcFile=src, jpgFileName=_jpgName)
    if os.path.exists(_jpg_file):
        s3.upload_file(_jpg_file, bucket, _key, ExtraArgs={'ACL': 'public-read', "StorageClass":"STANDARD"})
        logger.info("success upload_file: {0} end, use time {1}".format(_jpg_file, time.time() - t1))
        os.remove(_jpg_file)
    else:
        logger.info("fail upload_file: {0} end, use time {1}".format(_jpg_file, time.time() - t1))

def deldir(dir):
    shutil.rmtree(dir)
    os.mkdir(dir)

def main():
    tif_path = PIEConfig["tif_path"]
    key = PIEConfig["s3_key"]
    t1 = time.time()
    logger.info("running ...")

    _tile_aws = AWSTool(PIEConfig["tile_bucket"])
    _tile_s3 = _tile_aws.initS3Client()
    _tile_bucket = _tile_aws.getBucket()

    _files = walkDirFile(tif_path, ".tif")
    _file_info = {}
    for _file in _files:
        t2 = time.time()
        logger.info(f"process file: {_file}")
        #### TODO：强制将影像转为3857投影 ####
        _raw_file_name = os.path.basename(_file)
        _file_name = os.path.splitext(_raw_file_name)[0]
        _file_path = os.path.dirname(_file)
        _temp_file = os.path.join(_file_path, f"temp-{_raw_file_name}")
        if os.path.exists(_temp_file):
            os.remove(_temp_file)
        transformTif(_file, _temp_file)
        if os.path.exists(_temp_file):
            os.remove(_file)
        os.system(f"mv {_temp_file} {_file}")
        #####################################

        _file_names = _file_name.split("_")
        _folder_name = "_".join(_file_names[:len(_file_names) - 1])
        if _folder_name not in _file_info:
            _file_info[_folder_name] = []
        _file_info[_folder_name].append(_file)

        # 切瓦片
        generateTileToS3(s3=_tile_s3,
                         bucket=_tile_bucket,
                         key=f"{key}/{_folder_name}",
                         tif=_file,
                         epsg=3857,
                         compress=True,
                         nodata=0)
        logger.info(f"process use time: {time.time() - t2}")

    for _k, _v in _file_info.items():
        # 写入ES
        writeToES(files=_v, key=f"{key}/{_k}/{_k}", bucket=_tile_bucket)

    logger.info(f"end, use time : {time.time() - t1}")
    deldir(tif_path)

if __name__ == "__main__":
    main()

